{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6d82f19a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_processing import get_from_file\n",
    "from word_cloud import display_word_cloud\n",
    "from topic_analysis import do_lda, print_top_words\n",
    "from clustering import do_kmeans, visualize_kmeans\n",
    "from classificiation import do_svm\n",
    "import datetime as dt\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import joblib\n",
    "\n",
    "# period\n",
    "begin_date = dt.datetime(2022, 2, 24)\n",
    "end_date = dt.datetime(2022, 4, 30)\n",
    "period = \"2022.02.24-2022.04.30\"\n",
    "lang = \"en\"\n",
    "\n",
    "# tokenize\n",
    "weibo_data_path = \"data/weibo_2022.02.01-2022.11.30.jsonl\"\n",
    "twitter_data_path = \"data/twitter_2022.02.24-2022.11.26.jsonl\"\n",
    "processed_data_path = \"data/%s_tokenized_%s.xlsx\" % (period, lang)\n",
    "\n",
    "# word cloud\n",
    "word_cloud_mask_path = \"data/plane.png\"\n",
    "word_cloud_file_path = \"out/word_cloud_%s_%s.png\" % (period, lang)\n",
    "\n",
    "# topic analysis\n",
    "feature_word_cnt = 500\n",
    "topic_cnt = 13\n",
    "topic_top_word_cnt = 25\n",
    "topic_analysis_res_path = \"out/lda_%d_%d.html\" % (feature_word_cnt, topic_cnt)\n",
    "lda_path = \"out/lda_%s_%s.mdl\" % (period, lang)\n",
    "feature_name_path = \"out/feature_name_%s_%s.mdl\" % (period, lang)\n",
    "\n",
    "# clustering\n",
    "cluster_cnt = 10\n",
    "kmeans_res_path = \"out/kmeans_%s_%d_%s.png\" % (period, cluster_cnt, lang)\n",
    "tendency_res_path = \"out/tendency_%s_%s.png\" % (period, lang)\n",
    "reduced_dimension = 2\n",
    "\n",
    "# classification\n",
    "kernel = \"poly\"\n",
    "clf_path = \"out/classifier_%s_%s.mdl\" % (period, lang)\n",
    "confusion_matrix_path = \"out/confusion_matrix_%s_%s_%s.png\" % (period, kernel, lang)\n",
    "\n",
    "data = get_from_file(weibo_data_path, twitter_data_path, processed_data_path, lang=lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "03e1aa6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>content</th>\n",
       "      <th>tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-02-24 06:48:02.000000</td>\n",
       "      <td>Ukraine MP Sophia Fedyna tells about the groun...</td>\n",
       "      <td>ground situation conversation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-02-24 06:48:02.000000</td>\n",
       "      <td>Footage of the airport bombing in Ivano-Franki...</td>\n",
       "      <td>Footage airport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-02-24 06:48:03.000000</td>\n",
       "      <td>A cruise missile fired by the Russian army fel...</td>\n",
       "      <td>cruise missile army</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-02-24 06:48:03.000000</td>\n",
       "      <td>🇺🇦 53rd Mechanized Brigade continues to suffer...</td>\n",
       "      <td>area</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-02-24 06:48:03.000000</td>\n",
       "      <td>Footage of the airport bombing in Ivano-Franki...</td>\n",
       "      <td>Footage airport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2961</th>\n",
       "      <td>2022-04-30 00:00:11.000000</td>\n",
       "      <td>Never assume you're safe in a war zone.\\n\\nThi...</td>\n",
       "      <td>war zone soldier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2962</th>\n",
       "      <td>2022-04-30 00:00:12.000000</td>\n",
       "      <td>#Democrats: \"What do you mean by #MAGA? When w...</td>\n",
       "      <td>regime country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2963</th>\n",
       "      <td>2022-04-30 00:00:13.000000</td>\n",
       "      <td>Sunrise in Kyiv. Ukrainians made it thru the 6...</td>\n",
       "      <td>night claim aid war 💛💙</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2964</th>\n",
       "      <td>2022-04-30 00:00:14.000000</td>\n",
       "      <td>#SlavaUkraini🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦Opinion: Expect Put...</td>\n",
       "      <td>SlavaUkraini🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦Opinion announcement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2965</th>\n",
       "      <td>2022-04-30 00:00:15.000000</td>\n",
       "      <td>⚠️ Over the past few days, there have been no ...</td>\n",
       "      <td>⚠️ flee bring emergency aid update</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2966 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            time  \\\n",
       "0     2022-02-24 06:48:02.000000   \n",
       "1     2022-02-24 06:48:02.000000   \n",
       "2     2022-02-24 06:48:03.000000   \n",
       "3     2022-02-24 06:48:03.000000   \n",
       "4     2022-02-24 06:48:03.000000   \n",
       "...                          ...   \n",
       "2961  2022-04-30 00:00:11.000000   \n",
       "2962  2022-04-30 00:00:12.000000   \n",
       "2963  2022-04-30 00:00:13.000000   \n",
       "2964  2022-04-30 00:00:14.000000   \n",
       "2965  2022-04-30 00:00:15.000000   \n",
       "\n",
       "                                                content  \\\n",
       "0     Ukraine MP Sophia Fedyna tells about the groun...   \n",
       "1     Footage of the airport bombing in Ivano-Franki...   \n",
       "2     A cruise missile fired by the Russian army fel...   \n",
       "3     🇺🇦 53rd Mechanized Brigade continues to suffer...   \n",
       "4     Footage of the airport bombing in Ivano-Franki...   \n",
       "...                                                 ...   \n",
       "2961  Never assume you're safe in a war zone.\\n\\nThi...   \n",
       "2962  #Democrats: \"What do you mean by #MAGA? When w...   \n",
       "2963  Sunrise in Kyiv. Ukrainians made it thru the 6...   \n",
       "2964  #SlavaUkraini🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦Opinion: Expect Put...   \n",
       "2965  ⚠️ Over the past few days, there have been no ...   \n",
       "\n",
       "                                           tokenized  \n",
       "0                      ground situation conversation  \n",
       "1                                    Footage airport  \n",
       "2                                cruise missile army  \n",
       "3                                               area  \n",
       "4                                    Footage airport  \n",
       "...                                              ...  \n",
       "2961                                war zone soldier  \n",
       "2962                                  regime country  \n",
       "2963                          night claim aid war 💛💙  \n",
       "2964  SlavaUkraini🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦🇺🇦Opinion announcement  \n",
       "2965              ⚠️ flee bring emergency aid update  \n",
       "\n",
       "[2966 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1afb7db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyLDAvis\n",
    "import pyLDAvis.sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7b842dd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_vectorizer = CountVectorizer(strip_accents='unicode',\n",
    "                                    max_features=feature_word_cnt,\n",
    "                                    stop_words='english',\n",
    "                                    max_df=0.5,\n",
    "                                    min_df=10)\n",
    "tf = tf_vectorizer.fit_transform(data[\"tokenized\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e11c2fb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 171.19041290056367 -32738.914961720217\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [9]\u001b[0m, in \u001b[0;36m<cell line: 4>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      5\u001b[0m lda \u001b[38;5;241m=\u001b[39m LatentDirichletAllocation(n_components\u001b[38;5;241m=\u001b[39mi, max_iter\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50\u001b[39m,\n\u001b[0;32m      6\u001b[0m                                 learning_method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbatch\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m      7\u001b[0m                                 learning_offset\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50\u001b[39m,random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m      8\u001b[0m lda\u001b[38;5;241m.\u001b[39mfit(tf)\n\u001b[1;32m----> 9\u001b[0m plexs\u001b[38;5;241m.\u001b[39mappend(\u001b[43mlda\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mperplexity\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtf\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m     10\u001b[0m scores\u001b[38;5;241m.\u001b[39mappend(lda\u001b[38;5;241m.\u001b[39mscore(tf))\n\u001b[0;32m     11\u001b[0m \u001b[38;5;28mprint\u001b[39m(i, plexs[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m], scores[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m])\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:891\u001b[0m, in \u001b[0;36mLatentDirichletAllocation.perplexity\u001b[1;34m(self, X, sub_sampling)\u001b[0m\n\u001b[0;32m    887\u001b[0m check_is_fitted(\u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m    888\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_non_neg_array(\n\u001b[0;32m    889\u001b[0m     X, reset_n_features\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, whom\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLatentDirichletAllocation.perplexity\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    890\u001b[0m )\n\u001b[1;32m--> 891\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_perplexity_precomp_distr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msub_sampling\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msub_sampling\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:843\u001b[0m, in \u001b[0;36mLatentDirichletAllocation._perplexity_precomp_distr\u001b[1;34m(self, X, doc_topic_distr, sub_sampling)\u001b[0m\n\u001b[0;32m    822\u001b[0m \u001b[38;5;124;03m\"\"\"Calculate approximate perplexity for data X with ability to accept\u001b[39;00m\n\u001b[0;32m    823\u001b[0m \u001b[38;5;124;03mprecomputed doc_topic_distr\u001b[39;00m\n\u001b[0;32m    824\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    840\u001b[0m \u001b[38;5;124;03m    Perplexity score.\u001b[39;00m\n\u001b[0;32m    841\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    842\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m doc_topic_distr \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m--> 843\u001b[0m     doc_topic_distr \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_unnormalized_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    844\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    845\u001b[0m     n_samples, n_components \u001b[38;5;241m=\u001b[39m doc_topic_distr\u001b[38;5;241m.\u001b[39mshape\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:689\u001b[0m, in \u001b[0;36mLatentDirichletAllocation._unnormalized_transform\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    676\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_unnormalized_transform\u001b[39m(\u001b[38;5;28mself\u001b[39m, X):\n\u001b[0;32m    677\u001b[0m     \u001b[38;5;124;03m\"\"\"Transform data X according to fitted model.\u001b[39;00m\n\u001b[0;32m    678\u001b[0m \n\u001b[0;32m    679\u001b[0m \u001b[38;5;124;03m    Parameters\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    687\u001b[0m \u001b[38;5;124;03m        Document topic distribution for X.\u001b[39;00m\n\u001b[0;32m    688\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 689\u001b[0m     doc_topic_distr, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_e_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcal_sstats\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrandom_init\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    691\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m doc_topic_distr\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:446\u001b[0m, in \u001b[0;36mLatentDirichletAllocation._e_step\u001b[1;34m(self, X, cal_sstats, random_init, parallel)\u001b[0m\n\u001b[0;32m    444\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m parallel \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    445\u001b[0m     parallel \u001b[38;5;241m=\u001b[39m Parallel(n_jobs\u001b[38;5;241m=\u001b[39mn_jobs, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mmax\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbose \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m--> 446\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mparallel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    447\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdelayed\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_update_doc_distribution\u001b[49m\u001b[43m)\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    448\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx_slice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    449\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexp_dirichlet_component_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    450\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdoc_topic_prior_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    451\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_doc_update_iter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    452\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmean_change_tol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    453\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcal_sstats\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    454\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    455\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    456\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43midx_slice\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mgen_even_slices\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mn_jobs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    457\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    459\u001b[0m \u001b[38;5;66;03m# merge result\u001b[39;00m\n\u001b[0;32m    460\u001b[0m doc_topics, sstats_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mresults)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\parallel.py:1085\u001b[0m, in \u001b[0;36mParallel.__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m   1076\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1077\u001b[0m     \u001b[38;5;66;03m# Only set self._iterating to True if at least a batch\u001b[39;00m\n\u001b[0;32m   1078\u001b[0m     \u001b[38;5;66;03m# was dispatched. In particular this covers the edge\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1082\u001b[0m     \u001b[38;5;66;03m# was very quick and its callback already dispatched all the\u001b[39;00m\n\u001b[0;32m   1083\u001b[0m     \u001b[38;5;66;03m# remaining jobs.\u001b[39;00m\n\u001b[0;32m   1084\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iterating \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m-> 1085\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdispatch_one_batch\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[0;32m   1086\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iterating \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_original_iterator \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1088\u001b[0m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdispatch_one_batch(iterator):\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\parallel.py:901\u001b[0m, in \u001b[0;36mParallel.dispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    899\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m    900\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 901\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dispatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtasks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    902\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\parallel.py:819\u001b[0m, in \u001b[0;36mParallel._dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    817\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m    818\u001b[0m     job_idx \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jobs)\n\u001b[1;32m--> 819\u001b[0m     job \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_backend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply_async\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    820\u001b[0m     \u001b[38;5;66;03m# A job can complete so quickly than its callback is\u001b[39;00m\n\u001b[0;32m    821\u001b[0m     \u001b[38;5;66;03m# called before we get here, causing self._jobs to\u001b[39;00m\n\u001b[0;32m    822\u001b[0m     \u001b[38;5;66;03m# grow. To ensure correct results ordering, .insert is\u001b[39;00m\n\u001b[0;32m    823\u001b[0m     \u001b[38;5;66;03m# used (rather than .append) in the following line\u001b[39;00m\n\u001b[0;32m    824\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jobs\u001b[38;5;241m.\u001b[39minsert(job_idx, job)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\_parallel_backends.py:208\u001b[0m, in \u001b[0;36mSequentialBackend.apply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    206\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply_async\u001b[39m(\u001b[38;5;28mself\u001b[39m, func, callback\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m    207\u001b[0m     \u001b[38;5;124;03m\"\"\"Schedule a func to be run\"\"\"\u001b[39;00m\n\u001b[1;32m--> 208\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mImmediateResult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    209\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m callback:\n\u001b[0;32m    210\u001b[0m         callback(result)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\_parallel_backends.py:597\u001b[0m, in \u001b[0;36mImmediateResult.__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    594\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, batch):\n\u001b[0;32m    595\u001b[0m     \u001b[38;5;66;03m# Don't delay the application, to avoid keeping the input\u001b[39;00m\n\u001b[0;32m    596\u001b[0m     \u001b[38;5;66;03m# arguments in memory\u001b[39;00m\n\u001b[1;32m--> 597\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresults \u001b[38;5;241m=\u001b[39m \u001b[43mbatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\parallel.py:288\u001b[0m, in \u001b[0;36mBatchedCalls.__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    284\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    285\u001b[0m     \u001b[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001b[39;00m\n\u001b[0;32m    286\u001b[0m     \u001b[38;5;66;03m# change the default number of processes to -1\u001b[39;00m\n\u001b[0;32m    287\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m parallel_backend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backend, n_jobs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_n_jobs):\n\u001b[1;32m--> 288\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m [func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    289\u001b[0m                 \u001b[38;5;28;01mfor\u001b[39;00m func, args, kwargs \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mitems]\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\joblib\\parallel.py:288\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    284\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    285\u001b[0m     \u001b[38;5;66;03m# Set the default nested backend to self._backend but do not set the\u001b[39;00m\n\u001b[0;32m    286\u001b[0m     \u001b[38;5;66;03m# change the default number of processes to -1\u001b[39;00m\n\u001b[0;32m    287\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m parallel_backend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backend, n_jobs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_n_jobs):\n\u001b[1;32m--> 288\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m [func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m    289\u001b[0m                 \u001b[38;5;28;01mfor\u001b[39;00m func, args, kwargs \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mitems]\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\utils\\fixes.py:117\u001b[0m, in \u001b[0;36m_FuncWrapper.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    116\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m config_context(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig):\n\u001b[1;32m--> 117\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\sklearn\\decomposition\\_lda.py:124\u001b[0m, in \u001b[0;36m_update_doc_distribution\u001b[1;34m(X, exp_topic_word_distr, doc_topic_prior, max_doc_update_iter, mean_change_tol, cal_sstats, random_state)\u001b[0m\n\u001b[0;32m    120\u001b[0m \u001b[38;5;66;03m# The optimal phi_{dwk} is proportional to\u001b[39;00m\n\u001b[0;32m    121\u001b[0m \u001b[38;5;66;03m# exp(E[log(theta_{dk})]) * exp(E[log(beta_{dw})]).\u001b[39;00m\n\u001b[0;32m    122\u001b[0m norm_phi \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mdot(exp_doc_topic_d, exp_topic_word_d) \u001b[38;5;241m+\u001b[39m EPS\n\u001b[1;32m--> 124\u001b[0m doc_topic_d \u001b[38;5;241m=\u001b[39m exp_doc_topic_d \u001b[38;5;241m*\u001b[39m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcnts\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m/\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mnorm_phi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexp_topic_word_d\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mT\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    125\u001b[0m \u001b[38;5;66;03m# Note: adds doc_topic_prior to doc_topic_d, in-place.\u001b[39;00m\n\u001b[0;32m    126\u001b[0m _dirichlet_expectation_1d(doc_topic_d, doc_topic_prior, exp_doc_topic_d)\n",
      "File \u001b[1;32m<__array_function__ internals>:177\u001b[0m, in \u001b[0;36mdot\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "plexs = []\n",
    "scores = []\n",
    "n_max_topics = 15\n",
    "for i in range(1,n_max_topics):\n",
    "    lda = LatentDirichletAllocation(n_components=i, max_iter=50,\n",
    "                                    learning_method='batch',\n",
    "                                    learning_offset=50,random_state=0)\n",
    "    lda.fit(tf)\n",
    "    plexs.append(lda.perplexity(tf))\n",
    "    scores.append(lda.score(tf))\n",
    "    print(i, plexs[-1], scores[-1])\n",
    "    pic = pyLDAvis.sklearn.prepare(lda, tf, tf_vectorizer)\n",
    "    topic_analysis_res_path = \"out/lda_%d_%d_%s.html\" % (feature_word_cnt, i, lang)\n",
    "    pyLDAvis.save_html(pic, topic_analysis_res_path)\n",
    "    \n",
    "    \n",
    "n_t = n_max_topics - 1  # 区间最右侧的值。注意：不能大于n_max_topics\n",
    "x = list(range(1, n_t + 1))\n",
    "plt.plot(x, plexs[0:n_t])\n",
    "plt.xlabel(\"number of topics\")\n",
    "plt.ylabel(\"perplexity\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "362a23c3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
